{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.03179106, -0.03966626,  0.02588559, -0.03134463], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoyElEQVR4nO3df3RU9Z3/8ddMfoxAmIkBkkkkQRQKBgh2AcOsrbUlJfzQyhr3qGUFuxw4somnGmsxXapi9xhX96w/ugp/7K6450ixdEVXKlgECWsNqClZfmkKHNpgySQozUyC5ud8vn/45Z6OIjAhZD4zeT7Ouedk7ucz977v54STF/d+7r0uY4wRAACARdzxLgAAAOCLCCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDpxDSjPPvusLr/8cl1yySUqLi7Wu+++G89yAACAJeIWUF566SVVVlbqoYce0u9+9ztNnTpVpaWlamlpiVdJAADAEq54vSywuLhYM2bM0L/9279JkiKRiPLz83X33XfrgQceiEdJAADAEqnx2GlXV5fq6upUVVXlrHO73SopKVFtbe2X+nd2dqqzs9P5HIlEdPLkSY0YMUIul2tAagYAABfGGKO2tjbl5eXJ7T77RZy4BJSPP/5Yvb29ysnJiVqfk5OjDz/88Ev9q6urtWrVqoEqDwAAXETHjh3T6NGjz9onLgElVlVVVaqsrHQ+h0IhFRQU6NixY/J6vXGsDAAAnK9wOKz8/HwNHz78nH3jElBGjhyplJQUNTc3R61vbm6W3+//Un+PxyOPx/Ol9V6vl4ACAECCOZ/pGXG5iyc9PV3Tpk3Ttm3bnHWRSETbtm1TIBCIR0kAAMAicbvEU1lZqcWLF2v69Om65ppr9NRTT+nUqVP6wQ9+EK+SAACAJeIWUG699VadOHFCDz74oILBoK6++mpt2bLlSxNnAQDA4BO356BciHA4LJ/Pp1AoxBwUAAASRCx/v3kXDwAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdfo9oDz88MNyuVxRy8SJE532jo4OlZeXa8SIEcrIyFBZWZmam5v7uwwAAJDALsoZlEmTJqmpqclZ3n77baft3nvv1WuvvaYNGzaopqZGx48f180333wxygAAAAkq9aJsNDVVfr//S+tDoZD+4z/+Q+vWrdN3vvMdSdLzzz+vq666Srt27dLMmTMvRjkAACDBXJQzKIcOHVJeXp6uuOIKLVy4UI2NjZKkuro6dXd3q6SkxOk7ceJEFRQUqLa29iu319nZqXA4HLUAAIDk1e8Bpbi4WGvXrtWWLVu0evVqHT16VN/85jfV1tamYDCo9PR0ZWZmRn0nJydHwWDwK7dZXV0tn8/nLPn5+f1dNgAAsEi/X+KZO3eu83NRUZGKi4s1ZswY/fKXv9SQIUP6tM2qqipVVlY6n8PhMCEFAIAkdtFvM87MzNTXvvY1HT58WH6/X11dXWptbY3q09zcfMY5K6d5PB55vd6oBQAAJK+LHlDa29t15MgR5ebmatq0aUpLS9O2bduc9oaGBjU2NioQCFzsUgAAQILo90s8P/rRj3TjjTdqzJgxOn78uB566CGlpKTo9ttvl8/n05IlS1RZWamsrCx5vV7dfffdCgQC3MEDAAAc/R5QPvroI91+++365JNPNGrUKH3jG9/Qrl27NGrUKEnSk08+KbfbrbKyMnV2dqq0tFTPPfdcf5cBAAASmMsYY+JdRKzC4bB8Pp9CoRDzUQAASBCx/P3mXTwAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOvEHFB27typG2+8UXl5eXK5XHrllVei2o0xevDBB5Wbm6shQ4aopKREhw4diupz8uRJLVy4UF6vV5mZmVqyZIna29sv6EAAAEDyiDmgnDp1SlOnTtWzzz57xvbHH39czzzzjNasWaPdu3dr2LBhKi0tVUdHh9Nn4cKFOnDggLZu3apNmzZp586dWrZsWd+PAgAAJBWXMcb0+csulzZu3KgFCxZI+vzsSV5enu677z796Ec/kiSFQiHl5ORo7dq1uu222/TBBx+osLBQ7733nqZPny5J2rJli+bNm6ePPvpIeXl559xvOByWz+dTKBSS1+vta/kAAGAAxfL3u1/noBw9elTBYFAlJSXOOp/Pp+LiYtXW1kqSamtrlZmZ6YQTSSopKZHb7dbu3bvPuN3Ozk6Fw+GoBQAAJK9+DSjBYFCSlJOTE7U+JyfHaQsGg8rOzo5qT01NVVZWltPni6qrq+Xz+ZwlPz+/P8sGAACWSYi7eKqqqhQKhZzl2LFj8S4JAABcRP0aUPx+vySpubk5an1zc7PT5vf71dLSEtXe09OjkydPOn2+yOPxyOv1Ri0AACB59WtAGTt2rPx+v7Zt2+asC4fD2r17twKBgCQpEAiotbVVdXV1Tp/t27crEomouLi4P8sBAAAJKjXWL7S3t+vw4cPO56NHj6q+vl5ZWVkqKCjQPffco3/6p3/S+PHjNXbsWP30pz9VXl6ec6fPVVddpTlz5mjp0qVas2aNuru7VVFRodtuu+287uABAADJL+aA8v777+vb3/6287myslKStHjxYq1du1Y//vGPderUKS1btkytra36xje+oS1btuiSSy5xvvPiiy+qoqJCs2bNktvtVllZmZ555pl+OBwAAJAMLug5KPHCc1AAAEg8cXsOCgAAQH8goAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsE7MAWXnzp268cYblZeXJ5fLpVdeeSWq/c4775TL5Ypa5syZE9Xn5MmTWrhwobxerzIzM7VkyRK1t7df0IEAAIDkEXNAOXXqlKZOnapnn332K/vMmTNHTU1NzvKLX/wiqn3hwoU6cOCAtm7dqk2bNmnnzp1atmxZ7NUDAICklBrrF+bOnau5c+eetY/H45Hf7z9j2wcffKAtW7bovffe0/Tp0yVJP//5zzVv3jz9y7/8i/Ly8mItCQAAJJmLMgdlx44dys7O1oQJE7R8+XJ98sknTlttba0yMzOdcCJJJSUlcrvd2r179xm319nZqXA4HLUAAIDk1e8BZc6cOfqv//ovbdu2Tf/8z/+smpoazZ07V729vZKkYDCo7OzsqO+kpqYqKytLwWDwjNusrq6Wz+dzlvz8/P4uGwAAWCTmSzzncttttzk/T5kyRUVFRbryyiu1Y8cOzZo1q0/brKqqUmVlpfM5HA4TUgAASGIX/TbjK664QiNHjtThw4clSX6/Xy0tLVF9enp6dPLkya+ct+LxeOT1eqMWAACQvC56QPnoo4/0ySefKDc3V5IUCATU2tqquro6p8/27dsViURUXFx8scsBAAAJIOZLPO3t7c7ZEEk6evSo6uvrlZWVpaysLK1atUplZWXy+/06cuSIfvzjH2vcuHEqLS2VJF111VWaM2eOli5dqjVr1qi7u1sVFRW67bbbuIMHAABIklzGGBPLF3bs2KFvf/vbX1q/ePFirV69WgsWLNCePXvU2tqqvLw8zZ49Wz/72c+Uk5Pj9D158qQqKir02muvye12q6ysTM8884wyMjLOq4ZwOCyfz6dQKMTlHgAAEkQsf79jDig2IKAAAJB4Yvn7zbt4AACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6Mb8sEAAGQlvwsJr2bD5rn0sy/SoI/O0AVQRgIBFQAFjHGKOu9pMKNe47a7+ejlMDVBGAgcYlHgD2MUaR3p54VwEgjggoACxkZCK98S4CQBwRUABYxxgj09sd7zIAxBEBBYCVTC9nUIDBjIACwD7GKBJhDgowmBFQAFiISzzAYEdAAWCdz+egcAYFGMwIKAAsxCUeYLAjoACwjxHPQQEGOQIKAOsYcYkHGOwIKADsY3hQGzDYEVAAWCfS263P/nz87J1cbg3JyhuYggAMOAIKAOtEujv16Yk/nrWPy+3W8NyvDVBFAAYaAQVAgnLJlcIL2YFkRUABkLDcKWnxLgHARUJAAZCQXJLcqQQUIFkRUAAkJpdLbi7xAEmLgAIgYbm4xAMkrZgCSnV1tWbMmKHhw4crOztbCxYsUENDQ1Sfjo4OlZeXa8SIEcrIyFBZWZmam5uj+jQ2Nmr+/PkaOnSosrOzdf/996unh4cyAYiFi4ACJLGYAkpNTY3Ky8u1a9cubd26Vd3d3Zo9e7ZOnTrl9Ln33nv12muvacOGDaqpqdHx48d18803O+29vb2aP3++urq69M477+iFF17Q2rVr9eCDD/bfUQEYFNxuLvEAycpljDF9/fKJEyeUnZ2tmpoaXXfddQqFQho1apTWrVunW265RZL04Ycf6qqrrlJtba1mzpypzZs364YbbtDx48eVk5MjSVqzZo1WrFihEydOKD09/Zz7DYfD8vl8CoVC8nq9fS0fgKU6Qi3at37lWfu40zya/LcPyzN8xABVBeBCxfL3+4LmoIRCIUlSVlaWJKmurk7d3d0qKSlx+kycOFEFBQWqra2VJNXW1mrKlClOOJGk0tJShcNhHThw4Iz76ezsVDgcjloADHY8BwVIZn0OKJFIRPfcc4+uvfZaTZ48WZIUDAaVnp6uzMzMqL45OTkKBoNOn78MJ6fbT7edSXV1tXw+n7Pk5+f3tWwASYTnoADJq88Bpby8XPv379f69ev7s54zqqqqUigUcpZjx45d9H0CsJtLBBQgmfXp/GhFRYU2bdqknTt3avTo0c56v9+vrq4utba2Rp1FaW5ult/vd/q8++67Uds7fZfP6T5f5PF45PF4+lIqgARz3tPiXFziAZJZTGdQjDGqqKjQxo0btX37do0dOzaqfdq0aUpLS9O2bducdQ0NDWpsbFQgEJAkBQIB7du3Ty0tLU6frVu3yuv1qrCw8EKOBUCSiPTy2AFgsIvpvx/l5eVat26dXn31VQ0fPtyZM+Lz+TRkyBD5fD4tWbJElZWVysrKktfr1d13361AIKCZM2dKkmbPnq3CwkLdcccdevzxxxUMBrVy5UqVl5dzlgSAJMn0dse7BABxFlNAWb16tSTp+uuvj1r//PPP684775QkPfnkk3K73SorK1NnZ6dKS0v13HPPOX1TUlK0adMmLV++XIFAQMOGDdPixYv1yCOPXNiRAEgaEQIKMOhd0HNQ4oXnoADJyxij8PEP9ftNT561X0r6EH39zqfkcrkGqDIAF2rAnoMCABdDpIczKMBgR0ABYB3moAAgoACwDmdQABBQAFiHSbIACCgArBPp6Yp3CQDijIACwDrMQQFAQAFgGcOTZAEQUADYhzMoAAgoAOxijE58+NtzdhsxfuYAFAMgXggoAKzT09F+zj5pwy4dgEoAxAsBBUBCcqemxbsEABcRAQVAQnKnEFCAZEZAAZCQOIMCJDcCCoCE5E5Jj3cJAC4iAgqAhOTiDAqQ1AgoABKSOyU13iUAuIgIKAASkjuVSzxAMiOgAEhI3MUDJDcCCoCExF08QHIjoABISNzFAyQ3AgqAhOTiEg+Q1AgoAKxijDmvftzFAyQ3AgoAq5jenvPr6JJcLtfFLQZA3BBQAFgl0tsd7xIAWICAAsAqnweU87vMAyB5EVAAWCXSwxkUAAQUAJYxXOIBIAIKAMtwBgWAREABYBkmyQKQCCgALMMlHgASAQWAZSI9XdzEAyC2gFJdXa0ZM2Zo+PDhys7O1oIFC9TQ0BDV5/rrr5fL5Ypa7rrrrqg+jY2Nmj9/voYOHars7Gzdf//96uk5z4czAUhqkfN9UBuApBbTs6JrampUXl6uGTNmqKenRz/5yU80e/ZsHTx4UMOGDXP6LV26VI888ojzeejQoc7Pvb29mj9/vvx+v9555x01NTVp0aJFSktL06OPPtoPhwQgkTEHBYAUY0DZsmVL1Oe1a9cqOztbdXV1uu6665z1Q4cOld/vP+M2fvOb3+jgwYN68803lZOTo6uvvlo/+9nPtGLFCj388MNKT+cNpcBgFunpincJACxwQXNQQqGQJCkrKytq/YsvvqiRI0dq8uTJqqqq0qeffuq01dbWasqUKcrJyXHWlZaWKhwO68CBA2fcT2dnp8LhcNQCIDl1hk/oXJNQ0oZdKpeLKXRAMuvz60AjkYjuueceXXvttZo8ebKz/vvf/77GjBmjvLw87d27VytWrFBDQ4NefvllSVIwGIwKJ5Kcz8Fg8Iz7qq6u1qpVq/paKoAE0np0zzn7XHr51XK5eZsxkMz6/C+8vLxc+/fv19tvvx21ftmyZc7PU6ZMUW5urmbNmqUjR47oyiuv7NO+qqqqVFlZ6XwOh8PKz8/vW+EAEp4rJVXiRcZAUuvTOdKKigpt2rRJb731lkaPHn3WvsXFxZKkw4cPS5L8fr+am5uj+pz+/FXzVjwej7xeb9QCYPByp6SKhAIkt5gCijFGFRUV2rhxo7Zv366xY8ee8zv19fWSpNzcXElSIBDQvn371NLS4vTZunWrvF6vCgsLYykHwCDlSkmLdwkALrKYLvGUl5dr3bp1evXVVzV8+HBnzojP59OQIUN05MgRrVu3TvPmzdOIESO0d+9e3XvvvbruuutUVFQkSZo9e7YKCwt1xx136PHHH1cwGNTKlStVXl4uj8fT/0cIIOm4U9LEGRQgucV0BmX16tUKhUK6/vrrlZub6ywvvfSSJCk9PV1vvvmmZs+erYkTJ+q+++5TWVmZXnvtNWcbKSkp2rRpk1JSUhQIBPR3f/d3WrRoUdRzUwDgbNwpqXKRT4CkFtMZFGPOfutffn6+ampqzrmdMWPG6PXXX49l1wDgcKVyiQdIdjxIAEDCcbu5xAMkOwIKgITjSk0jnwBJjoACIOF8fpsxgGRGQAGQcLiLB0h+BBQACYfH3APJj4ACwBrGmHO8JvBzLi7xAEmPgALAGqa3R+d6k7H0+cUdFw9CAZIaAQWANSKRHukcz1sCMDgQUABY4/MzKABAQAFgEdPbc84nVgMYHAgoAKwR6e3W+cxBAZD8CCgArBGJ9DIHBYAkAgoAizAHBcBpBBQA1jAR5qAA+BwBBYA1TG83l3gASCKgALBI5Dwf1AYg+RFQAFiD24wBnEZAAWCNCJNkAfx/BBQA1mhr+r0i3R1n7TN01BilDskYoIoAxAuvBAXQL4wx6u3tvaBtdLZ9IhM5+zbShmbKuFLV09P3sy0pKSm8bBCwHAEFQL84dOiQJk2adEHbqF76HX1r6piz9vnvlzfqqe9X6WTbZ33ah8fjUTgcJqAAliOgAOgXxpgLOqshSSZy7gmyXd296uru7vO+UlJS+vQ9AAOLgALAOj0mVc2dl+uzyHC5ZJSR8mdlp/9RLpfU09urCHf6AEmPgALAKsa49LvwbLX1jFCX8cglo3T3ZzrRna/JGW+ruydyXmdaACQ2AgoAa0Tk1q7Q99Taky3p8zkiRlJnJEMfdUyUWxF19x7gDAowCHCbMQBr1LfNigonf8nIrT92TNKR9okEFGAQIKAAsMzZ7q5xqbs3ogiXeICkR0ABkFB6eiK8TxAYBAgoABIKd/EAgwMBBYA1ijJ2KCPlzzrzG42NLvM0yJ96kEs8wCAQU0BZvXq1ioqK5PV65fV6FQgEtHnzZqe9o6ND5eXlGjFihDIyMlRWVqbm5uaobTQ2Nmr+/PkaOnSosrOzdf/991/ww50AJIdUV7e+kfkreVM+VqqrU1JELkWU5vpMuelHNCWjRpFIF2dQgEEgptuMR48erccee0zjx4+XMUYvvPCCbrrpJu3Zs0eTJk3Svffeq1//+tfasGGDfD6fKioqdPPNN+u3v/2tJKm3t1fz58+X3+/XO++8o6amJi1atEhpaWl69NFHL8oBAkgcuz/8SK2nOtRjDutPHeN1qvdSuRSRN/VjtV9ySH+QdPhPf453mQAGgMuYC/uvSFZWlp544gndcsstGjVqlNatW6dbbrlFkvThhx/qqquuUm1trWbOnKnNmzfrhhtu0PHjx5WTkyNJWrNmjVasWKETJ04oPT39vPYZDofl8/l05513nvd3AFxcoVBIL730UrzLOCe3260lS5bwLh4gDrq6urR27VqFQiF5vd6z9u3zg9p6e3u1YcMGnTp1SoFAQHV1deru7lZJSYnTZ+LEiSooKHACSm1traZMmeKEE0kqLS3V8uXLdeDAAX39618/4746OzvV2dnpfA6Hw5KkO+64QxkZvHYdsEFjY2NCBJSUlBQCChAn7e3tWrt27Xn1jTmg7Nu3T4FAQB0dHcrIyNDGjRtVWFio+vp6paenKzMzM6p/Tk6OgsGgJCkYDEaFk9Ptp9u+SnV1tVatWvWl9dOnTz9nAgMwMHw+X7xLOC9ut1szZsyQ2809AsBAO32C4XzE/C90woQJqq+v1+7du7V8+XItXrxYBw8ejHUzMamqqlIoFHKWY8eOXdT9AQCA+Ir5DEp6errGjRsnSZo2bZree+89Pf3007r11lvV1dWl1tbWqLMozc3N8vv9kiS/36933303anun7/I53edMPB6PPB5PrKUCAIAEdcHnOCORiDo7OzVt2jSlpaVp27ZtTltDQ4MaGxsVCAQkSYFAQPv27VNLS4vTZ+vWrfJ6vSosLLzQUgAAQJKI6QxKVVWV5s6dq4KCArW1tWndunXasWOH3njjDfl8Pi1ZskSVlZXKysqS1+vV3XffrUAgoJkzZ0qSZs+ercLCQt1xxx16/PHHFQwGtXLlSpWXl3OGBAAAOGIKKC0tLVq0aJGamprk8/lUVFSkN954Q9/97nclSU8++aTcbrfKysrU2dmp0tJSPffcc873U1JStGnTJi1fvlyBQEDDhg3T4sWL9cgjj/TvUQEAgIR2wc9BiYfTz0E5n/uoAQyMhoYGTZw4Md5lnJPH49Gnn37KXTxAHMTy95t/oQAAwDoEFAAAYB0CCgAAsA4BBQAAWKfP7+IBgL+UkZGhBQsWxLuMc0pLS4t3CQDOAwEFQL+47LLLtHHjxniXASBJcIkHAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwTkwBZfXq1SoqKpLX65XX61UgENDmzZud9uuvv14ulytqueuuu6K20djYqPnz52vo0KHKzs7W/fffr56env45GgAAkBRSY+k8evRoPfbYYxo/fryMMXrhhRd00003ac+ePZo0aZIkaenSpXrkkUec7wwdOtT5ube3V/Pnz5ff79c777yjpqYmLVq0SGlpaXr00Uf76ZAAAECicxljzIVsICsrS0888YSWLFmi66+/XldffbWeeuqpM/bdvHmzbrjhBh0/flw5OTmSpDVr1mjFihU6ceKE0tPTz2uf4XBYPp9PoVBIXq/3QsoHAAADJJa/332eg9Lb26v169fr1KlTCgQCzvoXX3xRI0eO1OTJk1VVVaVPP/3UaautrdWUKVOccCJJpaWlCofDOnDgwFfuq7OzU+FwOGoBAADJK6ZLPJK0b98+BQIBdXR0KCMjQxs3blRhYaEk6fvf/77GjBmjvLw87d27VytWrFBDQ4NefvllSVIwGIwKJ5Kcz8Fg8Cv3WV1drVWrVsVaKgAASFAxB5QJEyaovr5eoVBIv/rVr7R48WLV1NSosLBQy5Ytc/pNmTJFubm5mjVrlo4cOaIrr7yyz0VWVVWpsrLS+RwOh5Wfn9/n7QEAALvFfIknPT1d48aN07Rp01RdXa2pU6fq6aefPmPf4uJiSdLhw4clSX6/X83NzVF9Tn/2+/1fuU+Px+PcOXR6AQAAyeuCn4MSiUTU2dl5xrb6+npJUm5uriQpEAho3759amlpcfps3bpVXq/XuUwEAAAQ0yWeqqoqzZ07VwUFBWpra9O6deu0Y8cOvfHGGzpy5IjWrVunefPmacSIEdq7d6/uvfdeXXfddSoqKpIkzZ49W4WFhbrjjjv0+OOPKxgMauXKlSovL5fH47koBwgAABJPTAGlpaVFixYtUlNTk3w+n4qKivTGG2/ou9/9ro4dO6Y333xTTz31lE6dOqX8/HyVlZVp5cqVzvdTUlK0adMmLV++XIFAQMOGDdPixYujnpsCAABwwc9BiQeegwIAQOIZkOegAAAAXCwEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOqnxLqAvjDGSpHA4HOdKAADA+Tr9d/v03/GzSciA0tbWJknKz8+PcyUAACBWbW1t8vl8Z+3jMucTYywTiUTU0NCgwsJCHTt2TF6vN94lJaxwOKz8/HzGsR8wlv2HsewfjGP/YSz7hzFGbW1tysvLk9t99lkmCXkGxe1267LLLpMkeb1efln6AePYfxjL/sNY9g/Gsf8wlhfuXGdOTmOSLAAAsA4BBQAAWCdhA4rH49FDDz0kj8cT71ISGuPYfxjL/sNY9g/Gsf8wlgMvISfJAgCA5JawZ1AAAEDyIqAAAADrEFAAAIB1CCgAAMA6CRlQnn32WV1++eW65JJLVFxcrHfffTfeJVln586duvHGG5WXlyeXy6VXXnklqt0YowcffFC5ubkaMmSISkpKdOjQoag+J0+e1MKFC+X1epWZmaklS5aovb19AI8i/qqrqzVjxgwNHz5c2dnZWrBggRoaGqL6dHR0qLy8XCNGjFBGRobKysrU3Nwc1aexsVHz58/X0KFDlZ2drfvvv189PT0DeShxtXr1ahUVFTkPuQoEAtq8ebPTzhj23WOPPSaXy6V77rnHWcd4np+HH35YLpcrapk4caLTzjjGmUkw69evN+np6eY///M/zYEDB8zSpUtNZmamaW5ujndpVnn99dfNP/7jP5qXX37ZSDIbN26Man/ssceMz+czr7zyivm///s/873vfc+MHTvWfPbZZ06fOXPmmKlTp5pdu3aZ//3f/zXjxo0zt99++wAfSXyVlpaa559/3uzfv9/U19ebefPmmYKCAtPe3u70ueuuu0x+fr7Ztm2bef/9983MmTPNX//1XzvtPT09ZvLkyaakpMTs2bPHvP7662bkyJGmqqoqHocUF//zP/9jfv3rX5vf//73pqGhwfzkJz8xaWlpZv/+/cYYxrCv3n33XXP55ZeboqIi88Mf/tBZz3ien4ceeshMmjTJNDU1OcuJEyecdsYxvhIuoFxzzTWmvLzc+dzb22vy8vJMdXV1HKuy2xcDSiQSMX6/3zzxxBPOutbWVuPxeMwvfvELY4wxBw8eNJLMe++95/TZvHmzcblc5k9/+tOA1W6blpYWI8nU1NQYYz4ft7S0NLNhwwanzwcffGAkmdraWmPM52HR7XabYDDo9Fm9erXxer2ms7NzYA/AIpdeeqn593//d8awj9ra2sz48ePN1q1bzbe+9S0noDCe5++hhx4yU6dOPWMb4xh/CXWJp6urS3V1dSopKXHWud1ulZSUqLa2No6VJZajR48qGAxGjaPP51NxcbEzjrW1tcrMzNT06dOdPiUlJXK73dq9e/eA12yLUCgkScrKypIk1dXVqbu7O2osJ06cqIKCgqixnDJlinJycpw+paWlCofDOnDgwABWb4fe3l6tX79ep06dUiAQYAz7qLy8XPPnz48aN4nfyVgdOnRIeXl5uuKKK7Rw4UI1NjZKYhxtkFAvC/z444/V29sb9csgSTk5Ofrwww/jVFXiCQaDknTGcTzdFgwGlZ2dHdWempqqrKwsp89gE4lEdM899+jaa6/V5MmTJX0+Tunp6crMzIzq+8WxPNNYn24bLPbt26dAIKCOjg5lZGRo48aNKiwsVH19PWMYo/Xr1+t3v/ud3nvvvS+18Tt5/oqLi7V27VpNmDBBTU1NWrVqlb75zW9q//79jKMFEiqgAPFUXl6u/fv36+233453KQlpwoQJqq+vVygU0q9+9SstXrxYNTU18S4r4Rw7dkw//OEPtXXrVl1yySXxLiehzZ071/m5qKhIxcXFGjNmjH75y19qyJAhcawMUoLdxTNy5EilpKR8aRZ1c3Oz/H5/nKpKPKfH6mzj6Pf71dLSEtXe09OjkydPDsqxrqio0KZNm/TWW29p9OjRznq/36+uri61trZG9f/iWJ5prE+3DRbp6ekaN26cpk2bpurqak2dOlVPP/00Yxijuro6tbS06K/+6q+Umpqq1NRU1dTU6JlnnlFqaqpycnIYzz7KzMzU1772NR0+fJjfSwskVEBJT0/XtGnTtG3bNmddJBLRtm3bFAgE4lhZYhk7dqz8fn/UOIbDYe3evdsZx0AgoNbWVtXV1Tl9tm/frkgkouLi4gGvOV6MMaqoqNDGjRu1fft2jR07Nqp92rRpSktLixrLhoYGNTY2Ro3lvn37ogLf1q1b5fV6VVhYODAHYqFIJKLOzk7GMEazZs3Svn37VF9f7yzTp0/XwoULnZ8Zz75pb2/XkSNHlJuby++lDeI9SzdW69evNx6Px6xdu9YcPHjQLFu2zGRmZkbNosbnM/z37Nlj9uzZYySZf/3XfzV79uwxf/zjH40xn99mnJmZaV599VWzd+9ec9NNN53xNuOvf/3rZvfu3ebtt98248ePH3S3GS9fvtz4fD6zY8eOqFsRP/30U6fPXXfdZQoKCsz27dvN+++/bwKBgAkEAk776VsRZ8+eberr682WLVvMqFGjBtWtiA888ICpqakxR48eNXv37jUPPPCAcblc5je/+Y0xhjG8UH95F48xjOf5uu+++8yOHTvM0aNHzW9/+1tTUlJiRo4caVpaWowxjGO8JVxAMcaYn//856agoMCkp6eba665xuzatSveJVnnrbfeMpK+tCxevNgY8/mtxj/96U9NTk6O8Xg8ZtasWaahoSFqG5988om5/fbbTUZGhvF6veYHP/iBaWtri8PRxM+ZxlCSef75550+n332mfmHf/gHc+mll5qhQ4eav/mbvzFNTU1R2/nDH/5g5s6da4YMGWJGjhxp7rvvPtPd3T3ARxM/f//3f2/GjBlj0tPTzahRo8ysWbOccGIMY3ihvhhQGM/zc+utt5rc3FyTnp5uLrvsMnPrrbeaw4cPO+2MY3y5jDEmPuduAAAAziyh5qAAAIDBgYACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOv8P1hT9mRWLvciAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.4261, 0.5739],\n",
       "         [0.4948, 0.5052]], grad_fn=<SoftmaxBackward0>),)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "model_action = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model_action(torch.randn(2, 4)),"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1050, -0.4790],\n",
       "        [ 0.1700, -0.3048]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_value = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    ")\n",
    "\n",
    "model_value_next = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 2),\n",
    ")\n",
    "\n",
    "model_value_next.load_state_dict(model_value.state_dict())\n",
    "\n",
    "model_value(torch.randn(2, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 4)\n",
    "    prob = model_action(state)\n",
    "\n",
    "    #根据概率选择一个动作\n",
    "    action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]\n",
    "\n",
    "    return action\n",
    "\n",
    "\n",
    "get_action([1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((224, 0), 224)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    old_count = len(datas)\n",
    "\n",
    "    #玩到新增了N个数据为止\n",
    "    while len(datas) - old_count < 200:\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "            #记录数据样本\n",
    "            datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "    update_count = len(datas) - old_count\n",
    "    drop_count = max(len(datas) - 10000, 0)\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 10000:\n",
    "        datas.pop(0)\n",
    "\n",
    "    return update_count, drop_count\n",
    "\n",
    "\n",
    "update_data(), len(datas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2456/1417522126.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-7.6131e-02, -6.2511e-01,  1.1829e-01,  1.0267e+00],\n",
       "         [ 8.3266e-04, -3.6003e-02,  5.0907e-02,  3.7703e-02],\n",
       "         [ 5.7219e-02,  2.4526e-01, -4.3534e-02, -3.1517e-01],\n",
       "         [-7.6134e-02, -6.4104e-01,  2.0860e-01,  1.3629e+00],\n",
       "         [ 8.5525e-02,  6.4917e-01, -1.5286e-01, -1.1831e+00]]),\n",
       " tensor([[1],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1]]),\n",
       " tensor([[1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.],\n",
       "         [1.]]),\n",
       " tensor([[-8.8633e-02, -4.3174e-01,  1.3882e-01,  7.7338e-01],\n",
       "         [ 1.1260e-04, -2.3182e-01,  5.1661e-02,  3.4600e-01],\n",
       "         [ 6.2124e-02,  5.0786e-02, -4.9838e-02, -3.6528e-02],\n",
       "         [-8.8955e-02, -8.3808e-01,  2.3585e-01,  1.7129e+00],\n",
       "         [ 9.8509e-02,  8.4591e-01, -1.7652e-01, -1.5196e+00]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [1],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 4]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    action = torch.LongTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 4]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 4)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state[:5], action[:5], reward[:5], next_state[:5], over[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_update(model, model_next):\n",
    "    for param, param_next in zip(model.parameters(), model_next.parameters()):\n",
    "        #以一个小的比例更新\n",
    "        value = param_next.data * 0.995 + param.data * 0.005\n",
    "        param_next.data.copy_(value)\n",
    "\n",
    "\n",
    "soft_update(torch.nn.Linear(4, 64), torch.nn.Linear(4, 64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_target(reward, next_state, over):\n",
    "    #计算动作的概率\n",
    "    #[b, 4] -> [b, 2]\n",
    "    prob = model_action(next_state)\n",
    "\n",
    "    #计算动作的熵\n",
    "    #[b, 2]\n",
    "    entropy = prob * torch.log(prob + 1e-8)\n",
    "\n",
    "    #所有动作的熵求和\n",
    "    #[b, 2] -> [b, 1]\n",
    "    entropy = -entropy.sum(dim=1, keepdim=True)\n",
    "\n",
    "    #计算next_state的价值\n",
    "    #[b, 4] -> [b, 2]\n",
    "    target = model_value_next(next_state)\n",
    "\n",
    "    #[b, 2] * [b, 2] -> [b, 2]\n",
    "    target = (prob * target)\n",
    "\n",
    "    #[b, 2] -> [b, 1]\n",
    "    target = target.sum(dim=1, keepdim=True)\n",
    "\n",
    "    #这里的操作是在target上加上了动作的熵\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target = target + 0.002 * entropy\n",
    "\n",
    "    #[b, 2]\n",
    "    target *= 0.98\n",
    "    target *= (1 - over)\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_target(reward, next_state, over).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #评估state的价值\n",
    "    value = model_value(state)\n",
    "\n",
    "    #取出对应动作的分数\n",
    "    value = value.gather(1, action)\n",
    "\n",
    "    return value\n",
    "\n",
    "\n",
    "get_value(state, action).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0558, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_action(state):\n",
    "    #计算动作的概率\n",
    "    #[b, 4] -> [b, 2]\n",
    "    prob = model_action(state)\n",
    "\n",
    "    #计算动作的熵\n",
    "    #[b, 2]\n",
    "    entropy = prob * (prob + 1e-8).log()\n",
    "\n",
    "    #所有动作的熵求和\n",
    "    #[b, 2] -> [b, 1]\n",
    "    entropy = -entropy.sum(dim=1, keepdim=True)\n",
    "\n",
    "    #评估state的价值\n",
    "    #[b, 4] -> [b, 2]\n",
    "    value = model_value(state)\n",
    "\n",
    "    #按动作的概率对价值加权\n",
    "    #[b, 2] * [b, 2] -> [b, 2]\n",
    "    value *= prob\n",
    "\n",
    "    #所有动作的价值求和\n",
    "    #[b, 2] -> [b, 1]\n",
    "    value = value.sum(dim=1, keepdim=True)\n",
    "\n",
    "    #这里的操作是在target上加上了动作的熵,这个值越大越好\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    loss_action = value + 0.002 * entropy\n",
    "\n",
    "    #因为是计算loss,所以对这个值符号取反\n",
    "    return -loss_action.mean()\n",
    "\n",
    "\n",
    "get_loss_action(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 435 10.1\n",
      "20 5473 200.0\n",
      "40 10000 195.3\n",
      "60 10000 185.7\n",
      "80 10000 196.9\n",
      "100 10000 184.8\n",
      "120 10000 200.0\n",
      "140 10000 200.0\n",
      "160 10000 200.0\n",
      "180 10000 107.6\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer_action = torch.optim.Adam(model_action.parameters(), lr=1e-3)\n",
    "    optimizer_value = torch.optim.Adam(model_value.parameters(), lr=1e-2)\n",
    "\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算target,这个target里已经考虑了动作的熵\n",
    "            #[b, 1]\n",
    "            target = get_target(reward, next_state, over)\n",
    "            target = target.detach()\n",
    "\n",
    "            #计算value\n",
    "            value = get_value(state, action)\n",
    "\n",
    "            #计算loss,value的目标是要贴近target\n",
    "            loss_value = loss_fn(value, target)\n",
    "\n",
    "            #更新参数\n",
    "            optimizer_value.zero_grad()\n",
    "            loss_value.backward()\n",
    "            optimizer_value.step()\n",
    "\n",
    "            #使用model_value计算model_action的loss\n",
    "            loss_action = get_loss_action(state)\n",
    "            optimizer_action.zero_grad()\n",
    "            loss_action.backward()\n",
    "            optimizer_action.step()\n",
    "\n",
    "            #增量更新next模型\n",
    "            soft_update(model_value, model_value_next)\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, len(datas), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmy0lEQVR4nO3dfXBUVZ7/8U9CkuaxOwZIOpEEURggQtAFDL3OuMySIUBkZIxV6rCSmaWgZBN/A3EYzCwj4mwZF7fGh1mFX9XuiltlhhFLdGUEJwYJ6xgejGR40qxQzAaHdIKy6YZoQpI+vz8s7m9bUenQ0Kfb96vqVqXvOffe7z2Vqv7UuQ+dZIwxAgAAsEhyrAsAAAD4PAIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALBOTAPK008/rWuuuUYDBw5UYWGh9u7dG8tyAACAJWIWUH7729+qsrJSa9as0bvvvqspU6aouLhY7e3tsSoJAABYIilWPxZYWFio6dOn65//+Z8lSaFQSLm5ubrvvvv0wAMPxKIkAABgiZRYHPTcuXNqbGxUVVWVsy45OVlFRUVqaGj4Qv/u7m51d3c7n0OhkE6fPq3hw4crKSnpitQMAAAujTFGZ86cUU5OjpKTv/oiTkwCykcffaS+vj5lZWWFrc/KytL777//hf7V1dVau3btlSoPAABcRidOnNCoUaO+sk9MAkqkqqqqVFlZ6XwOBALKy8vTiRMn5Ha7Y1gZAAC4WMFgULm5uRo2bNjX9o1JQBkxYoQGDBigtra2sPVtbW3yer1f6O9yueRyub6w3u12E1AAAIgzF3N7Rkye4klLS9PUqVNVV1fnrAuFQqqrq5PP54tFSQAAwCIxu8RTWVmpsrIyTZs2TTfddJOeeOIJdXZ26sc//nGsSgIAAJaIWUC58847derUKT344IPy+/264YYbtH379i/cOAsAAL55YvYelEsRDAbl8XgUCAS4BwUAgDgRyfc3v8UDAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGCdqAeUhx56SElJSWHLhAkTnPauri6Vl5dr+PDhGjp0qEpLS9XW1hbtMgAAQBy7LDMo119/vVpbW53lrbfectpWrFihV199VZs3b1Z9fb1Onjyp22+//XKUAQAA4lTKZdlpSoq8Xu8X1gcCAf3rv/6rampq9Nd//deSpGeffVYTJ07U7t27NWPGjMtRDgAAiDOXZQblgw8+UE5Ojq699lotXLhQLS0tkqTGxkb19PSoqKjI6TthwgTl5eWpoaHhS/fX3d2tYDAYtgAAgMQV9YBSWFiojRs3avv27Vq/fr2OHz+u73znOzpz5oz8fr/S0tKUnp4etk1WVpb8fv+X7rO6uloej8dZcnNzo102AACwSNQv8cydO9f5u6CgQIWFhRo9erReeOEFDRo0qF/7rKqqUmVlpfM5GAwSUgAASGCX/THj9PR0fetb39LRo0fl9Xp17tw5dXR0hPVpa2u74D0r57lcLrnd7rAFAAAkrsseUM6ePatjx44pOztbU6dOVWpqqurq6pz25uZmtbS0yOfzXe5SAABAnIj6JZ6f/vSnmj9/vkaPHq2TJ09qzZo1GjBggO6++255PB4tXrxYlZWVysjIkNvt1n333Sefz8cTPAAAwBH1gPLhhx/q7rvv1scff6yRI0fq29/+tnbv3q2RI0dKkh5//HElJyertLRU3d3dKi4u1jPPPBPtMgAAQBxLMsaYWBcRqWAwKI/Ho0AgwP0oAADEiUi+v/ktHgAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdSIOKLt27dL8+fOVk5OjpKQkvfzyy2Htxhg9+OCDys7O1qBBg1RUVKQPPvggrM/p06e1cOFCud1upaena/HixTp79uwlnQgAAEgcEQeUzs5OTZkyRU8//fQF29etW6ennnpKGzZs0J49ezRkyBAVFxerq6vL6bNw4UIdPnxYtbW12rp1q3bt2qWlS5f2/ywAAEBCSTLGmH5vnJSkLVu2aMGCBZI+mz3JycnR/fffr5/+9KeSpEAgoKysLG3cuFF33XWX3nvvPeXn52vfvn2aNm2aJGn79u2aN2+ePvzwQ+Xk5HztcYPBoDwejwKBgNxud3/LBwAAV1Ak399RvQfl+PHj8vv9KioqctZ5PB4VFhaqoaFBktTQ0KD09HQnnEhSUVGRkpOTtWfPngvut7u7W8FgMGwBAACJK6oBxe/3S5KysrLC1mdlZTltfr9fmZmZYe0pKSnKyMhw+nxedXW1PB6Ps+Tm5kazbAAAYJm4eIqnqqpKgUDAWU6cOBHrkgAAwGUU1YDi9XolSW1tbWHr29ranDav16v29vaw9t7eXp0+fdrp83kul0tutztsAQAAiSuqAWXMmDHyer2qq6tz1gWDQe3Zs0c+n0+S5PP51NHRocbGRqfPjh07FAqFVFhYGM1yAABAnEqJdIOzZ8/q6NGjzufjx4+rqalJGRkZysvL0/Lly/UP//APGjdunMaMGaNf/OIXysnJcZ70mThxoubMmaMlS5Zow4YN6unpUUVFhe66666LeoIHAAAkvogDyjvvvKPvfve7zufKykpJUllZmTZu3Kif/exn6uzs1NKlS9XR0aFvf/vb2r59uwYOHOhs8/zzz6uiokKzZs1ScnKySktL9dRTT0XhdAAAQCK4pPegxArvQQEAIP7E7D0oAAAA0UBAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgnYgDyq5duzR//nzl5OQoKSlJL7/8clj7j370IyUlJYUtc+bMCetz+vRpLVy4UG63W+np6Vq8eLHOnj17SScCAAASR8QBpbOzU1OmTNHTTz/9pX3mzJmj1tZWZ/nNb34T1r5w4UIdPnxYtbW12rp1q3bt2qWlS5dGXj0AAEhIKZFuMHfuXM2dO/cr+7hcLnm93gu2vffee9q+fbv27dunadOmSZJ+/etfa968efqnf/on5eTkRFoSAABIMJflHpSdO3cqMzNT48eP17Jly/Txxx87bQ0NDUpPT3fCiSQVFRUpOTlZe/bsueD+uru7FQwGwxYAAJC4oh5Q5syZo3//939XXV2d/vEf/1H19fWaO3eu+vr6JEl+v1+ZmZlh26SkpCgjI0N+v/+C+6yurpbH43GW3NzcaJcNAAAsEvElnq9z1113OX9PnjxZBQUFuu6667Rz507NmjWrX/usqqpSZWWl8zkYDBJSAABIYJf9MeNrr71WI0aM0NGjRyVJXq9X7e3tYX16e3t1+vTpL71vxeVyye12hy0AACBxXfaA8uGHH+rjjz9Wdna2JMnn86mjo0ONjY1Onx07digUCqmwsPBylwMAAOJAxJd4zp4968yGSNLx48fV1NSkjIwMZWRkaO3atSotLZXX69WxY8f0s5/9TGPHjlVxcbEkaeLEiZozZ46WLFmiDRs2qKenRxUVFbrrrrt4ggcAAEiSkowxJpINdu7cqe9+97tfWF9WVqb169drwYIF2r9/vzo6OpSTk6PZs2frl7/8pbKyspy+p0+fVkVFhV599VUlJyertLRUTz31lIYOHXpRNQSDQXk8HgUCAS73AAAQJyL5/o44oNiAgAIAQPyJ5Pub3+IBAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHXiOqCEQr2xLgEAAFwGcR1QTC8BBQCARBTfASXUF+sSAADAZRDXASXU1xPrEgAAwGUQ1wHFhEKxLgEAAFwGcR1QQn3cgwIAQCKK64BieIoHAICEREABAADWieuAwk2yAAAkprgOKKaPx4wBAEhEcR5QuMQDAEAiiuuAEuJFbQAAJKS4DihdHf5YlwAAAC6DuA4on3zUEusSAADAZRDXAQUAACQmAgoAALAOAQUAAFiHgAIAAKxDQAEAANaJKKBUV1dr+vTpGjZsmDIzM7VgwQI1NzeH9enq6lJ5ebmGDx+uoUOHqrS0VG1tbWF9WlpaVFJSosGDByszM1MrV65Uby8vXQMAAJ+JKKDU19ervLxcu3fvVm1trXp6ejR79mx1dnY6fVasWKFXX31VmzdvVn19vU6ePKnbb7/dae/r61NJSYnOnTunt99+W88995w2btyoBx98MHpnBQAA4lqSMcb0d+NTp04pMzNT9fX1uuWWWxQIBDRy5EjV1NTojjvukCS9//77mjhxohoaGjRjxgxt27ZNt956q06ePKmsrCxJ0oYNG7Rq1SqdOnVKaWlpX3vcYDAoj8ejd1/6lW78wYr+lg8AAK6g89/fgUBAbrf7K/te0j0ogUBAkpSRkSFJamxsVE9Pj4qKipw+EyZMUF5enhoaGiRJDQ0Nmjx5shNOJKm4uFjBYFCHDx++4HG6u7sVDAbDFgAAkLj6HVBCoZCWL1+um2++WZMmTZIk+f1+paWlKT09PaxvVlaW/H6/0+d/h5Pz7efbLqS6uloej8dZcnNz+1s2AACIA/0OKOXl5Tp06JA2bdoUzXouqKqqSoFAwFlOnDhx2Y8JAABiJ6U/G1VUVGjr1q3atWuXRo0a5az3er06d+6cOjo6wmZR2tra5PV6nT579+4N29/5p3zO9/k8l8sll8t1gRYjY4ySkpL6cxoAAMBSEc2gGGNUUVGhLVu2aMeOHRozZkxY+9SpU5Wamqq6ujpnXXNzs1paWuTz+SRJPp9PBw8eVHt7u9OntrZWbrdb+fn5kVVvjIwJRbYNAACwXkQzKOXl5aqpqdErr7yiYcOGOfeMeDweDRo0SB6PR4sXL1ZlZaUyMjLkdrt13333yefzacaMGZKk2bNnKz8/X/fcc4/WrVsnv9+v1atXq7y8/EtmSb6cMSEp1CclD4hoOwAAYLeIAsr69eslSTNnzgxb/+yzz+pHP/qRJOnxxx9XcnKySktL1d3dreLiYj3zzDNO3wEDBmjr1q1atmyZfD6fhgwZorKyMj388MORV2+MQqEQr8MFACDBXNJ7UGLl/HPU77xQrSnz71PKwCGxLgkAAHyNK/YelFgzJiQT6ot1GQAAIMriOqAoFJIxBBQAABJNXAcUZlAAAEhMcR5QjEyIx4wBAEg0cR1QFOqTCfXGugoAABBlcR1QPrvEwwwKAACJJs4DiuEeFAAAElBcBxRxkywAAAkprgNKV/CUOtv/FOsyAABAlMV1QDF9verr7Y51GQAAIMriOqAAAIDEREABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOvEf0AxRsaYWFcBAACiKO4DijEhiYACAEBCif+AEur7LKQAAICEkQABpZcZFAAAEkz8B5Q+ZlAAAEg08R9QQn0SAQUAgISSEAGFp3gAAEgsCRJQmEEBACCRJERA4SZZAAASSwIElF5mUAAASDARBZTq6mpNnz5dw4YNU2ZmphYsWKDm5uawPjNnzlRSUlLYcu+994b1aWlpUUlJiQYPHqzMzEytXLlSvb29/ToBEwpxkywAAAkmJZLO9fX1Ki8v1/Tp09Xb26uf//znmj17to4cOaIhQ4Y4/ZYsWaKHH37Y+Tx48GDn776+PpWUlMjr9ertt99Wa2urFi1apNTUVD3yyCMRn8BnMyhc4gEAIJFEFFC2b98e9nnjxo3KzMxUY2OjbrnlFmf94MGD5fV6L7iP3//+9zpy5IjeeOMNZWVl6YYbbtAvf/lLrVq1Sg899JDS0tIiOoFQX58UYgYFAIBEckn3oAQCAUlSRkZG2Prnn39eI0aM0KRJk1RVVaVPPvnEaWtoaNDkyZOVlZXlrCsuLlYwGNThw4cveJzu7m4Fg8Gw5bzerjPq6+2+lNMAAACWiWgG5X8LhUJavny5br75Zk2aNMlZ/8Mf/lCjR49WTk6ODhw4oFWrVqm5uVkvvfSSJMnv94eFE0nOZ7/ff8FjVVdXa+3atRds++SjFvV0BqSMq/t7KgAAwDL9Dijl5eU6dOiQ3nrrrbD1S5cudf6ePHmysrOzNWvWLB07dkzXXXddv45VVVWlyspK53MwGFRubm7/CgcAANbr1yWeiooKbd26VW+++aZGjRr1lX0LCwslSUePHpUkeb1etbW1hfU5//nL7ltxuVxyu91hCwAASFwRBRRjjCoqKrRlyxbt2LFDY8aM+dptmpqaJEnZ2dmSJJ/Pp4MHD6q9vd3pU1tbK7fbrfz8/EjKAQAACSqiSzzl5eWqqanRK6+8omHDhjn3jHg8Hg0aNEjHjh1TTU2N5s2bp+HDh+vAgQNasWKFbrnlFhUUFEiSZs+erfz8fN1zzz1at26d/H6/Vq9erfLycrlcruifIQAAiDsRzaCsX79egUBAM2fOVHZ2trP89re/lSSlpaXpjTfe0OzZszVhwgTdf//9Ki0t1auvvursY8CAAdq6dasGDBggn8+nv/mbv9GiRYvC3psCAAC+2SKaQfm6F6Ll5uaqvr7+a/czevRovfbaa5EcGgAAfIPE/W/xAACAxENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgkRUIzM1z5hBAAA4kdiBJS+3liXAAAAoihBAkqPJGZQAABIFAkRUEJ9veQTAAASSEIEFBPqFQkFAIDEkRABJdTbI+6RBQAgcSRGQOljBgUAgESSEAGFSzwAACSWxAgo3CQLAEBCSYiAEuIxYwAAEkpCBBTT10s8AQAggSREQAn19YjHeAAASBwJEVB41T0AAIklIQJK8M/vKdR7LtZlAACAKEmJdQHRcO7safX2nFNSav9nUpKSkjRgwIAoVgUAAPorIQKKJI0ZM0YfBz/t9/bz58/XSy+9FMWKAABAfyVMQOnt7VVvb/9nUPr6+qJYDQAAuBRxfQ/Kia7x6uxzx7oMAAAQZXE9g/LBJ9N0OqlXk4buinUpAAAgiuJ6BqXPpCrYO1L7AiXqS2YmBQCARBHXAeW8HjNQ5f9nc6zLAAAAUZIQAUX67DFhAACQGBImoAAAgMRBQAEAANZJiICSrF7932fuiXUZAAAgSiIKKOvXr1dBQYHcbrfcbrd8Pp+2bdvmtHd1dam8vFzDhw/X0KFDVVpaqra2trB9tLS0qKSkRIMHD1ZmZqZWrlzZ7xesJatXg5MDKvRsVai77es3AAAAcSGi96CMGjVKjz76qMaNGydjjJ577jnddttt2r9/v66//nqtWLFCv/vd77R582Z5PB5VVFTo9ttv1x/+8AdJn72ttaSkRF6vV2+//bZaW1u1aNEipaam6pFHHom4+CN/fFGjhrTqf1L+R13n+EVjAAASRZIxxlzKDjIyMvTYY4/pjjvu0MiRI1VTU6M77rhDkvT+++9r4sSJamho0IwZM7Rt2zbdeuutOnnypLKysiRJGzZs0KpVq3Tq1CmlpaVd1DGDwaA8Hs+llP0Fo0ePVnFxcVT3CQAA/r9z585p48aNCgQCcru/+v1l/X6TbF9fnzZv3qzOzk75fD41Njaqp6dHRUVFTp8JEyYoLy/PCSgNDQ2aPHmyE04kqbi4WMuWLdPhw4d14403XvBY3d3d6u7udj4Hg8H+lv2l8vLytHjx4qjvFwAAfObs2bPauHHjRfWNOKAcPHhQPp9PXV1dGjp0qLZs2aL8/Hw1NTUpLS1N6enpYf2zsrLk9/slSX6/PyycnG8/3/ZlqqurtXbt2khLjchVV12lm2666bIeAwCAb7JIJhgifopn/Pjxampq0p49e7Rs2TKVlZXpyJEjke4mIlVVVQoEAs5y4sSJy3o8AAAQWxHPoKSlpWns2LGSpKlTp2rfvn168skndeedd+rcuXPq6OgIm0Vpa2uT1+uVJHm9Xu3duzdsf+ef8jnf50JcLpdcLlekpQIAgDh1ye9BCYVC6u7u1tSpU5Wamqq6ujqnrbm5WS0tLfL5fJIkn8+ngwcPqr293elTW1srt9ut/Pz8Sy0FAAAkiIhmUKqqqjR37lzl5eXpzJkzqqmp0c6dO/X666/L4/Fo8eLFqqysVEZGhtxut+677z75fD7NmDFDkjR79mzl5+frnnvu0bp16+T3+7V69WqVl5czQwIAABwRBZT29nYtWrRIra2t8ng8Kigo0Ouvv67vfe97kqTHH39cycnJKi0tVXd3t4qLi/XMM8842w8YMEBbt27VsmXL5PP5NGTIEJWVlenhhx+O7lkBAIC4dsnvQYmFy/EelO9///t65ZVXorpPAADw/53//r6Y96AkxG/xAACAxEJAAQAA1iGgAAAA6xBQAACAdfr9Wzw2KCkpUWpqalT2xWvuAQCwR1wHlJqamq+9CxgAAMQfLvEAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWiSigrF+/XgUFBXK73XK73fL5fNq2bZvTPnPmTCUlJYUt9957b9g+WlpaVFJSosGDByszM1MrV65Ub29vdM4GAAAkhJRIOo8aNUqPPvqoxo0bJ2OMnnvuOd12223av3+/rr/+eknSkiVL9PDDDzvbDB482Pm7r69PJSUl8nq9evvtt9Xa2qpFixYpNTVVjzzySJROCQAAxLskY4y5lB1kZGToscce0+LFizVz5kzdcMMNeuKJJy7Yd9u2bbr11lt18uRJZWVlSZI2bNigVatW6dSpU0pLS7uoYwaDQXk8HgUCAbnd7kspHwAAXCGRfH/3+x6Uvr4+bdq0SZ2dnfL5fM76559/XiNGjNCkSZNUVVWlTz75xGlraGjQ5MmTnXAiScXFxQoGgzp8+PCXHqu7u1vBYDBsAQAAiSuiSzySdPDgQfl8PnV1dWno0KHasmWL8vPzJUk//OEPNXr0aOXk5OjAgQNatWqVmpub9dJLL0mS/H5/WDiR5Hz2+/1feszq6mqtXbs20lIBAECcijigjB8/Xk1NTQoEAnrxxRdVVlam+vp65efna+nSpU6/yZMnKzs7W7NmzdKxY8d03XXX9bvIqqoqVVZWOp+DwaByc3P7vT8AAGC3iC/xpKWlaezYsZo6daqqq6s1ZcoUPfnkkxfsW1hYKEk6evSoJMnr9aqtrS2sz/nPXq/3S4/pcrmcJ4fOLwAAIHFd8ntQQqGQuru7L9jW1NQkScrOzpYk+Xw+HTx4UO3t7U6f2tpaud1u5zIRAABARJd4qqqqNHfuXOXl5enMmTOqqanRzp079frrr+vYsWOqqanRvHnzNHz4cB04cEArVqzQLbfcooKCAknS7NmzlZ+fr3vuuUfr1q2T3+/X6tWrVV5eLpfLdVlOEAAAxJ+IAkp7e7sWLVqk1tZWeTweFRQU6PXXX9f3vvc9nThxQm+88YaeeOIJdXZ2Kjc3V6WlpVq9erWz/YABA7R161YtW7ZMPp9PQ4YMUVlZWdh7UwAAAC75PSixwHtQAACIP1fkPSgAAACXCwEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALBOSqwL6A9jjCQpGAzGuBIAAHCxzn9vn/8e/ypxGVDOnDkjScrNzY1xJQAAIFJnzpyRx+P5yj5J5mJijGVCoZCam5uVn5+vEydOyO12x7qkuBUMBpWbm8s4RgFjGT2MZXQwjtHDWEaHMUZnzpxRTk6OkpO/+i6TuJxBSU5O1tVXXy1Jcrvd/LNEAeMYPYxl9DCW0cE4Rg9jeem+bubkPG6SBQAA1iGgAAAA68RtQHG5XFqzZo1cLlesS4lrjGP0MJbRw1hGB+MYPYzllReXN8kCAIDEFrczKAAAIHERUAAAgHUIKAAAwDoEFAAAYJ24DChPP/20rrnmGg0cOFCFhYXau3dvrEuyzq5duzR//nzl5OQoKSlJL7/8cli7MUYPPvigsrOzNWjQIBUVFemDDz4I63P69GktXLhQbrdb6enpWrx4sc6ePXsFzyL2qqurNX36dA0bNkyZmZlasGCBmpubw/p0dXWpvLxcw4cP19ChQ1VaWqq2trawPi0tLSopKdHgwYOVmZmplStXqre390qeSkytX79eBQUFzkuufD6ftm3b5rQzhv336KOPKikpScuXL3fWMZ4X56GHHlJSUlLYMmHCBKedcYwxE2c2bdpk0tLSzL/927+Zw4cPmyVLlpj09HTT1tYW69Ks8tprr5m///u/Ny+99JKRZLZs2RLW/uijjxqPx2Nefvll88c//tF8//vfN2PGjDGffvqp02fOnDlmypQpZvfu3eY///M/zdixY83dd999hc8ktoqLi82zzz5rDh06ZJqamsy8efNMXl6eOXv2rNPn3nvvNbm5uaaurs688847ZsaMGeYv//Ivnfbe3l4zadIkU1RUZPbv329ee+01M2LECFNVVRWLU4qJ//iP/zC/+93vzH/913+Z5uZm8/Of/9ykpqaaQ4cOGWMYw/7au3evueaaa0xBQYH5yU9+4qxnPC/OmjVrzPXXX29aW1ud5dSpU0474xhbcRdQbrrpJlNeXu587uvrMzk5Oaa6ujqGVdnt8wElFAoZr9drHnvsMWddR0eHcblc5je/+Y0xxpgjR44YSWbfvn1On23btpmkpCTz5z//+YrVbpv29nYjydTX1xtjPhu31NRUs3nzZqfPe++9ZySZhoYGY8xnYTE5Odn4/X6nz/r1643b7Tbd3d1X9gQsctVVV5l/+Zd/YQz76cyZM2bcuHGmtrbW/NVf/ZUTUBjPi7dmzRozZcqUC7YxjrEXV5d4zp07p8bGRhUVFTnrkpOTVVRUpIaGhhhWFl+OHz8uv98fNo4ej0eFhYXOODY0NCg9PV3Tpk1z+hQVFSk5OVl79uy54jXbIhAISJIyMjIkSY2Njerp6QkbywkTJigvLy9sLCdPnqysrCynT3FxsYLBoA4fPnwFq7dDX1+fNm3apM7OTvl8Psawn8rLy1VSUhI2bhL/k5H64IMPlJOTo2uvvVYLFy5US0uLJMbRBnH1Y4EfffSR+vr6wv4ZJCkrK0vvv/9+jKqKP36/X5IuOI7n2/x+vzIzM8PaU1JSlJGR4fT5pgmFQlq+fLluvvlmTZo0SdJn45SWlqb09PSwvp8fywuN9fm2b4qDBw/K5/Opq6tLQ4cO1ZYtW5Sfn6+mpibGMEKbNm3Su+++q3379n2hjf/Ji1dYWKiNGzdq/Pjxam1t1dq1a/Wd73xHhw4dYhwtEFcBBYil8vJyHTp0SG+99VasS4lL48ePV1NTkwKBgF588UWVlZWpvr4+1mXFnRMnTugnP/mJamtrNXDgwFiXE9fmzp3r/F1QUKDCwkKNHj1aL7zwggYNGhTDyiDF2VM8I0aM0IABA75wF3VbW5u8Xm+Mqoo/58fqq8bR6/Wqvb09rL23t1enT5/+Ro51RUWFtm7dqjfffFOjRo1y1nu9Xp07d04dHR1h/T8/lhca6/Nt3xRpaWkaO3aspk6dqurqak2ZMkVPPvkkYxihxsZGtbe36y/+4i+UkpKilJQU1dfX66mnnlJKSoqysrIYz35KT0/Xt771LR09epT/SwvEVUBJS0vT1KlTVVdX56wLhUKqq6uTz+eLYWXxZcyYMfJ6vWHjGAwGtWfPHmccfT6fOjo61NjY6PTZsWOHQqGQCgsLr3jNsWKMUUVFhbZs2aIdO3ZozJgxYe1Tp05Vampq2Fg2NzerpaUlbCwPHjwYFvhqa2vldruVn59/ZU7EQqFQSN3d3YxhhGbNmqWDBw+qqanJWaZNm6aFCxc6fzOe/XP27FkdO3ZM2dnZ/F/aINZ36UZq06ZNxuVymY0bN5ojR46YpUuXmvT09LC7qPHZHf779+83+/fvN5LMr371K7N//37z3//938aYzx4zTk9PN6+88oo5cOCAue222y74mPGNN95o9uzZY9566y0zbty4b9xjxsuWLTMej8fs3Lkz7FHETz75xOlz7733mry8PLNjxw7zzjvvGJ/PZ3w+n9N+/lHE2bNnm6amJrN9+3YzcuTIb9SjiA888ICpr683x48fNwcOHDAPPPCASUpKMr///e+NMYzhpfrfT/EYw3herPvvv9/s3LnTHD9+3PzhD38wRUVFZsSIEaa9vd0YwzjGWtwFFGOM+fWvf23y8vJMWlqauemmm8zu3btjXZJ13nzzTSPpC0tZWZkx5rNHjX/xi1+YrKws43K5zKxZs0xzc3PYPj7++GNz9913m6FDhxq3221+/OMfmzNnzsTgbGLnQmMoyTz77LNOn08//dT83d/9nbnqqqvM4MGDzQ9+8APT2toatp8//elPZu7cuWbQoEFmxIgR5v777zc9PT1X+Gxi52//9m/N6NGjTVpamhk5cqSZNWuWE06MYQwv1ecDCuN5ce68806TnZ1t0tLSzNVXX23uvPNOc/ToUaedcYytJGOMic3cDQAAwIXF1T0oAADgm4GAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADr/D/oFnhKz7tDLAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "158.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
